package icu.smile;

import icu.smile.proxy.task.MiPuPageProcessor;
import icu.smile.proxy.task.MiPuPipeline;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * <p>
 * 爬去米扑代理
 * </p>
 *
 * @author starrysky
 * @since 2021/6/6
 */
@SpringBootApplication
public class MiPuProxyApplication {
    private static String URL = "https://proxy.mimvp.com/freeopen";
    public static void main(String[] args) {
        final ConfigurableApplicationContext ctx = SpringApplication.run(MiPuProxyApplication.class, args);
        System.setProperty("selenuim_config","src/main/resources/config.ini");
        Spider.create(new MiPuPageProcessor())
                .addUrl(URL)
                .setDownloader(new SeleniumDownloader())
                .addPipeline(ctx.getBean(MiPuPipeline.class))
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000000)))
                .thread(1).runAsync();
    }
}
